{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "TensorFlow_Keras_MNIST.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "v3YP5lGDptdj",
        "colab_type": "text"
      },
      "source": [
        "#Convert a Custom Model for MNIST Dataset to ONNX and Inference with ONNX Runtime\n",
        "\n",
        "This example performs a conversion of a 'hand-crafted' CNN for the MNIST dataset, loosely based on the lenet CNN. It creates a model in keras, evaluates it, converts it to ONNX format. Finnaly, it loads the created ONNX file to perform predictions and compare it againt keras' predictions. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "McVALi98usry",
        "colab_type": "code",
        "outputId": "d43e7bcb-4381-489f-b96c-8d0e2e9b8244",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "import sys\n",
        "import os\n",
        "import tensorflow as tf\n",
        "print(\"TensorFlow version is \"+tf.__version__)\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import time"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "TensorFlow version is 2.2.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NM4D1RxkS__x",
        "colab_type": "text"
      },
      "source": [
        "# Preparing the MNIST Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5UM_-1uGMlpo",
        "colab_type": "code",
        "outputId": "302d9b03-7eb5-4d3c-ffa1-00b745b11b0d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        }
      },
      "source": [
        "# Get MNIST dataset\n",
        "mnist = tf.keras.datasets.mnist\n",
        "# Load its data into training and test vectors\n",
        "(x_train, y_train),(x_test, y_test) = mnist.load_data()\n",
        "# Normalize the data\n",
        "x_train, x_test = x_train / 255.0, x_test / 255.0\n",
        "# Collect feature size info\n",
        "imgSize0=len(x_train[0])\n",
        "imgSize1=len(x_train[0][0])\n",
        "numPixels=imgSize0*imgSize1\n",
        "numTrainImages=len(x_train)\n",
        "featureShape=(1,imgSize0,imgSize1)\n",
        "\n",
        "print(\"Training dataset has \"+str(numTrainImages))\n",
        "print(\"Testing dataset has \"+str(len(x_test)))\n",
        "print(\"Feature shape is \"+str(featureShape))"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Training dataset has 60000\n",
            "Testing dataset has 10000\n",
            "Feature shape is (1, 28, 28)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uoWTP9Du0TTc",
        "colab_type": "code",
        "outputId": "d7e44927-5235-4869-b424-462db3044c3a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 363
        }
      },
      "source": [
        "train_labels_count = np.unique(y_train, return_counts=True)\n",
        "dataframe_train_labels = pd.DataFrame({'Label':train_labels_count[0], 'Count':train_labels_count[1]})\n",
        "dataframe_train_labels"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Label</th>\n",
              "      <th>Count</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>5923</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>6742</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>5958</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>6131</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>5842</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>5</td>\n",
              "      <td>5421</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>6</td>\n",
              "      <td>5918</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>7</td>\n",
              "      <td>6265</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>8</td>\n",
              "      <td>5851</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>9</td>\n",
              "      <td>5949</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Label  Count\n",
              "0      0   5923\n",
              "1      1   6742\n",
              "2      2   5958\n",
              "3      3   6131\n",
              "4      4   5842\n",
              "5      5   5421\n",
              "6      6   5918\n",
              "7      7   6265\n",
              "8      8   5851\n",
              "9      9   5949"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rY6mVyks1FsY",
        "colab_type": "code",
        "outputId": "448c6963-190c-40e0-901f-cabd4c118cd2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 363
        }
      },
      "source": [
        "test_labels_count = np.unique(y_test, return_counts=True)\n",
        "dataframe_test_labels = pd.DataFrame({'Label':test_labels_count[0], 'Count':test_labels_count[1]})\n",
        "dataframe_test_labels"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Label</th>\n",
              "      <th>Count</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>980</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>1135</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>1032</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>1010</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>982</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>5</td>\n",
              "      <td>892</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>6</td>\n",
              "      <td>958</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>7</td>\n",
              "      <td>1028</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>8</td>\n",
              "      <td>974</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>9</td>\n",
              "      <td>1009</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Label  Count\n",
              "0      0    980\n",
              "1      1   1135\n",
              "2      2   1032\n",
              "3      3   1010\n",
              "4      4    982\n",
              "5      5    892\n",
              "6      6    958\n",
              "7      7   1028\n",
              "8      8    974\n",
              "9      9   1009"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "biAtRdVyTHwd",
        "colab_type": "text"
      },
      "source": [
        "# Building or Loading the CNN Model with Keras\n",
        "\n",
        "If you are building a model in keras to convert it to ONNX, it's realy important that the first layer defines the ***input_shape***, otherwise you will see errors in the ONNX conversion. See [Issue #493](https://github.com/onnx/keras-onnx/issues/493) for more information.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1NO4CNgDBuNa",
        "colab_type": "code",
        "outputId": "13729bc4-c7eb-4795-b8e4-ddc1aaf12cf2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 278
        }
      },
      "source": [
        "# Clearup everything before running\n",
        "tf.keras.backend.clear_session()\n",
        "\n",
        "print ('creating a new model')\n",
        "# Clearup everything before running\n",
        "tf.keras.backend.clear_session()\n",
        "# Create model\n",
        "model = tf.keras.models.Sequential()\n",
        "# Add layers\n",
        "# The first layer MUST have input_shape. See the observation above.\n",
        "model.add(tf.keras.layers.Flatten(input_shape=(28, 28, 1)))\n",
        "model.add(tf.keras.layers.Dense(8, activation='relu'))\n",
        "model.add(tf.keras.layers.Dense(10, activation='softmax'))\n",
        "\n",
        "# Build model and print summary\n",
        "model.build(input_shape=featureShape)\n",
        "model.summary()"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "creating a new model\n",
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "flatten (Flatten)            (None, 784)               0         \n",
            "_________________________________________________________________\n",
            "dense (Dense)                (None, 8)                 6280      \n",
            "_________________________________________________________________\n",
            "dense_1 (Dense)              (None, 10)                90        \n",
            "=================================================================\n",
            "Total params: 6,370\n",
            "Trainable params: 6,370\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Yd3oPpYaxNen",
        "colab_type": "code",
        "outputId": "9e3e50f7-d146-416e-a53c-024804eddd7e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 938
        }
      },
      "source": [
        "model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])\n",
        "# Train\n",
        "print(\"Training Model\")\n",
        "# increase the number of epochs if you wish better accuracy\n",
        "history = model.fit(x_train, y_train, epochs=10)\n",
        "plt.plot(history.history[\"loss\"], color='r')\n",
        "plt.title(\"Loss\")\n",
        "plt.xlabel(\"Epoch\")\n",
        "plt.show()\n",
        "plt.plot(history.history[\"accuracy\"], color='b')\n",
        "plt.title(\"Accuracy\")\n",
        "plt.xlabel(\"Epoch\")\n",
        "plt.show()"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Training Model\n",
            "Epoch 1/10\n",
            "1875/1875 [==============================] - 2s 1ms/step - loss: 0.5394 - accuracy: 0.8426\n",
            "Epoch 2/10\n",
            "1875/1875 [==============================] - 2s 1ms/step - loss: 0.3159 - accuracy: 0.9092\n",
            "Epoch 3/10\n",
            "1875/1875 [==============================] - 2s 1ms/step - loss: 0.2891 - accuracy: 0.9165\n",
            "Epoch 4/10\n",
            "1875/1875 [==============================] - 2s 1ms/step - loss: 0.2713 - accuracy: 0.9224\n",
            "Epoch 5/10\n",
            "1875/1875 [==============================] - 2s 1ms/step - loss: 0.2592 - accuracy: 0.9263\n",
            "Epoch 6/10\n",
            "1875/1875 [==============================] - 2s 1ms/step - loss: 0.2510 - accuracy: 0.9285\n",
            "Epoch 7/10\n",
            "1875/1875 [==============================] - 2s 1ms/step - loss: 0.2441 - accuracy: 0.9310\n",
            "Epoch 8/10\n",
            "1875/1875 [==============================] - 2s 1ms/step - loss: 0.2391 - accuracy: 0.9325\n",
            "Epoch 9/10\n",
            "1875/1875 [==============================] - 2s 1ms/step - loss: 0.2343 - accuracy: 0.9337\n",
            "Epoch 10/10\n",
            "1875/1875 [==============================] - 2s 1ms/step - loss: 0.2305 - accuracy: 0.9346\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEWCAYAAABollyxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAfrUlEQVR4nO3deZSU9Z3v8feHZgc1II0ii90oi8QN7RDELs0enDiYM5mZaJYTb8xxzMjoRM+NZrlJLplzYuKcjJkMWYzXnEwmSoyZ3GCiUW+iiYoLjSKKSIR2A1FaEVdk/d4/flWhummggOp+qp/6vM6p0/UsVf2tPvB5nvo9v+f3U0RgZmb51S/rAszMrGc56M3Mcs5Bb2aWcw56M7Occ9CbmeWcg97MLOcc9GZmOeegt7om6SlJ78u6DrOe5KA3M8s5B71ZF5IGSbpK0nPFx1WSBhW3jZL0G0kbJW2QdJekfsVtl0laK+k1SSslvTfbT2KW9M+6ALMa9CVgJnAiEMCvgS8D/wu4FFgDNBb3nQmEpCnAXOAdEfGcpCagoXfLNuuez+jNdvVxYF5ErI+IDuB/A58sbtsKjAGOjIitEXFXpAGjtgODgGmSBkTEUxGxOpPqzbpw0Jvt6gjg6bLlp4vrAK4EVgG3SWqXdDlARKwC/hn4GrBe0gJJR2BWAxz0Zrt6DjiybHlCcR0R8VpEXBoRE4E5wCWltviIuC4iWouvDeCbvVu2Wfcc9GYwQNLg0gO4HviypEZJo4CvAP8FIOlMSUdLEvAKqclmh6Qpkt5TvGj7FrAJ2JHNxzHrzEFvBjeTgrn0GAy0AcuAR4AHgX8p7jsJ+H/A68C9wPci4g5S+/wVwIvA88Bo4Au99xHMdk+eeMTMLN98Rm9mlnMOejOznHPQm5nlnIPezCznKhoCQdJs4DukW7qviYgrumw/l3Qjydriqv+IiGuK27aTei4APBMRc/b0u0aNGhVNTU2V1m9mZsCSJUtejIjG7rbtNeglNQDzgfeTxvhYLGlhRDzWZdefR8Tcbt5iU0ScWGmxTU1NtLW1Vbq7mZkBkp7e3bZKmm5mAKsioj0itgALgLOqVZyZmfWsSoJ+LPBs2fKa4rquPiJpmaQbJY0vWz9YUpuk+yR9uLtfIOn84j5tHR0dlVdvZmZ7Va2LsTcBTRFxPHA78JOybUdGRAvwMeAqSUd1fXFEXB0RLRHR0tjYbROTmZntp0qCfi1QfoY+jp0XXQGIiJciYnNx8Rrg5LJta4s/24E7gekHUK+Zme2jSoJ+MTBJUrOkgcDZwMLyHSSNKVucA6worh9RPjMPcCrQ9SKumZn1oL32uomIbZLmAreSuldeGxHLJc0D2iJiIXCRpDnANmADcG7x5ccAP5S0g3RQuaKb3jpmZtaDam5Qs5aWlnD3SjOzfSNpSfF66C7yc2fshg0wbx48+GDWlZiZ1ZT8TA7e0ABf+xrs2AEnnZR1NWZmNSM/Z/SHHAInnAB33ZV1JWZmNSU/QQ9QKMB998HWrVlXYmZWM/IV9K2t8Oab8NBDWVdiZlYz8hf0AHffnW0dZmY1JF9Bf8QRMHGi2+nNzMrkK+ghtdPffTfU2P0BZmZZyV/Qt7bCiy/CypVZV2JmVhPyF/SFQvrpdnozMyCPQT95MjQ2up3ezKwof0EvpeYbn9GbmQF5DHpIQd/eDs89l3UlZmaZy2fQl9rp3XxjZpbToJ8+HYYNc/ONmRl5Dfr+/WHmTJ/Rm5mR16CH1HyzbBm88krWlZiZZSq/Qd/amu6OXbQo60rMzDKV36CfOTNNRuJ2ejOrc/kN+mHD0kxTbqc3szqX36CH1E7/wAOweXPWlZiZZSbfQd/amkK+rS3rSszMMpP/oAe305tZXaso6CXNlrRS0ipJl3ez/VxJHZKWFh+fKdv2KUlPFB+fqmbxe9XYCFOmuJ3ezOpa/73tIKkBmA+8H1gDLJa0MCIe67LrzyNibpfXjgS+CrQAASwpvvblqlRfiUIBbrwRduyAfvn+AmNm1p1Kkm8GsCoi2iNiC7AAOKvC9/8gcHtEbCiG++3A7P0rdT8VCrBxIyxf3qu/1sysVlQS9GOBZ8uW1xTXdfURScsk3Shp/L68VtL5ktoktXV0dFRYeoVK7fRuvjGzOlWttoybgKaIOJ501v6TfXlxRFwdES0R0dLY2Filkoqam9Ok4b4ga2Z1qpKgXwuML1seV1z3FxHxUkSUOqtfA5xc6Wt7XGkikrvu8oThZlaXKgn6xcAkSc2SBgJnAwvLd5A0pmxxDrCi+PxW4AOSRkgaAXyguK53FQqwZg0880yv/2ozs6zttddNRGyTNJcU0A3AtRGxXNI8oC0iFgIXSZoDbAM2AOcWX7tB0tdJBwuAeRGxoQc+x56Vt9MfeWSv/3ozsywpaqw5o6WlJdqqfSfr9u0wciSccw784AfVfW8zsxogaUlEtHS3rT46ljc0wKxZ7nljZnWpPoIeUjv9Y4/BSy9lXYmZWa+qn6AvtdPfc0+2dZiZ9bL6CfoZM2DgQPenN7O6Uz9BP3gwvOMdbqc3s7pTP0EPqfmmrQ3efDPrSszMek19BX2hANu2pVmnzMzqRH0F/axZaUgEN9+YWR2pr6AfMQKOPdYXZM2srtRX0ENqp1+0KDXhmJnVgfoL+kIBXn8dli3LuhIzs15Rf0HviUjMrM7UX9CPH59GsHQ7vZnVifoLevBEJGZWV+oz6AsFeOEFWL0660rMzHpc/QY9uJ3ezOpCfQb91KlpIhIHvZnVgfoM+n79Uju9L8iaWR2oz6CHFPRPPAHPP591JWZmPap+g77UTu+JSMws5+o36E86CYYMcTu9meVe/Qb9wIHwzne6nd7Mcq9+gx5SO/1DD8Frr2VdiZlZj6ko6CXNlrRS0ipJl+9hv49ICkktxeUmSZskLS0+flCtwquiUIAdO+C++7KuxMysx+w16CU1APOBM4BpwDmSpnWz30HAxcD9XTatjogTi48LqlBz9ZxySupq6XZ6M8uxSs7oZwCrIqI9IrYAC4Czutnv68A3gbeqWF/POuggOPFEt9ObWa5VEvRjgWfLltcU1/2FpJOA8RHx225e3yzpIUl/lFTo7hdIOl9Sm6S2jo6OSmuvjkIhNd1s2dK7v9fMrJcc8MVYSf2AbwOXdrN5HTAhIqYDlwDXSTq4604RcXVEtERES2Nj44GWtG9aW2HTpnRR1swshyoJ+rXA+LLlccV1JQcBxwJ3SnoKmAkslNQSEZsj4iWAiFgCrAYmV6PwqvFEJGaWc5UE/WJgkqRmSQOBs4GFpY0R8UpEjIqIpohoAu4D5kREm6TG4sVcJE0EJgHtVf8UB+Lww+Hoox30ZpZbew36iNgGzAVuBVYAN0TEcknzJM3Zy8tPA5ZJWgrcCFwQERsOtOiqKxTSUAg7dmRdiZlZ1SlqbJallpaWaGtr691feu21cN55sHw5TNul56iZWc2TtCQiWrrbVt93xpaUBjhzN0szyyEHPaQ2+tGj3U5vZrnkoAeQ0lm9z+jNLIcc9CWFAjz1FKxZk3UlZmZV5aAvKfWn91m9meWMg77khBNg+HC305tZ7jjoS/r3T6NZ+ozezHLGQV+uUIBHHoGNG7OuxMysahz05VpbIcIThptZrjjoy73znakJx803ZpYjDvpyQ4fCySf7gqyZ5YqDvqtCARYvhrf6zkRZZmZ74qDvqrU1zTa1eHHWlZiZVYWDvivfOGVmOeOg7+rQQ9NQxW6nN7OccNB3p7UVFi2C7duzrsTM7IA56LtTKMArr8Cjj2ZdiZnZAXPQd8ft9GaWIw767hx5JIwb53Z6M8sFB313pHRWf9ddaUgEM7M+zEG/O4UCPPdcmozEzKwPc9DvTqmd3s03ZtbHOeh359hj4ZBDfEHWzPq8ioJe0mxJKyWtknT5Hvb7iKSQ1FK27gvF162U9MFqFN0r+vWDU0/1Gb2Z9Xl7DXpJDcB84AxgGnCOpGnd7HcQcDFwf9m6acDZwNuB2cD3iu/XNxQK8Pjj0NGRdSVmZvutkjP6GcCqiGiPiC3AAuCsbvb7OvBNoHzYx7OABRGxOSKeBFYV369vKBTST09EYmZ9WCVBPxZ4tmx5TXHdX0g6CRgfEb/d19cWX3++pDZJbR21dPbc0gKDBrmd3sz6tAO+GCupH/Bt4NL9fY+IuDoiWiKipbGx8UBLqp5Bg2DGDLfTm1mfVknQrwXGly2PK64rOQg4FrhT0lPATGBh8YLs3l5b+1pb4cEH4Y03sq7EzGy/VBL0i4FJkpolDSRdXF1Y2hgRr0TEqIhoiogm4D5gTkS0Ffc7W9IgSc3AJOCBqn+KnlQowLZtcP/9e9/XzKwG7TXoI2IbMBe4FVgB3BARyyXNkzRnL69dDtwAPAb8DrgwIvrW2L+nnJKGRHA7vZn1UYoaG8ulpaUl2trasi6jsxNPhMZGuP32rCsxM+uWpCUR0dLdNt8ZW4nWVrj33tSEY2bWxzjoK1EopIuxS5dmXYmZ2T5z0FfCA5yZWR/moK/E2LHQ3OwLsmbWJznoK1UoeCISM+uTHPSVam1Ng5s98UTWlZiZ7RMHfaVKA5y5nd7M+hgHfaWmTIFRo9xOb2Z9joO+UuUThpuZ9SEO+n3R2gqrV8O6dVlXYmZWMQf9vii107v5xsz6EAf9vpg+HYYOdfONmfUpDvp9MWAAzJzpM3oz61Mc9PuqUICHH4ZXX826EjOzijjo91VrK+zYkUazNDPrAxz0+2rmTGhocDu9mfUZDvp9NXx4uijrdnoz6yMc9PujUEhzyG7enHUlZmZ75aDfH62t8NZb8OCDWVdiZrZXDvr94YlIzKwPcdDvj9GjYfJkt9ObWZ/goN9fhUIK+h07sq7EzGyPHPT7q7UVXn4ZVqzIuhIzsz2qKOglzZa0UtIqSZd3s/0CSY9IWirpbknTiuubJG0qrl8q6QfV/gCZ8UQkZtZH7DXoJTUA84EzgGnAOaUgL3NdRBwXEScC3wK+XbZtdUScWHxcUK3CMzdxIowZ46A3s5pXyRn9DGBVRLRHxBZgAXBW+Q4RUT7wyzAg/zNolyYi8QVZM6txlQT9WODZsuU1xXWdSLpQ0mrSGf1FZZuaJT0k6Y+SCt39AknnS2qT1NbR0bEP5WesUIBnnkkPM7MaVbWLsRExPyKOAi4DvlxcvQ6YEBHTgUuA6yQd3M1rr46IlohoaWxsrFZJPa/Un95n9WZWwyoJ+rXA+LLlccV1u7MA+DBARGyOiJeKz5cAq4HJ+1dqDTr+eDjoILfTm1lNqyToFwOTJDVLGgicDSws30HSpLLFDwFPFNc3Fi/mImkiMAlor0bhNaGhAWbN8hm9mdW0vQZ9RGwD5gK3AiuAGyJiuaR5kuYUd5srabmkpaQmmk8V158GLCuuvxG4ICI2VP1TZKlQgEcfhQ35+lhmlh/9K9kpIm4Gbu6y7itlzy/ezet+CfzyQAqseaV2+kWL4Mwzs63FzKwbvjP2QM2YkeaSdTu9mdUoB/2BGjIEWlrcTm9mNctBXw2FAixeDJs2ZV2JmdkuHPTVUCjA1q3wwANZV2JmtgsHfTXMmpV+uvnGzGqQg74aRo6EY4/1BVkzq0kO+mppbU1dLLdvz7oSM7NOHPTVUijAa6/BsmVZV2Jm1omDvlo8wJmZ1SgHfbVMmJAebqc3sxrjoK+m0kQkkf95V8ys73DQV1OhAOvWQXt+Bug0s77PQV9Nbqc3sxrkoK+madNgxAi305tZTXHQV1O/fums3kFvZjXEQV9tra3w5z/DypVZV2JmBjjoq++v/xoGD4bjjoMLL4Q1a7KuyMzqnIO+2o45Jp3Rn3ce/OhHcPTRcPHFqTeOmVkGHPQ9Yfx4+P73U+B/8pMwfz5MnAiXXgovvJB1dWZWZxz0PampKZ3Vr1wJZ58NV12VAv+yy+DFF7OuzszqhIO+Nxx1FPz4x7BiBfzN38CVV0JzM3zpS7BhQ9bVmVnOOeh70+TJ8NOfwvLlcOaZ8I1vpLP+r34VNm7MujozyykHfRaOOQauvz4NafzBD8K8eSnwv/51ePXVrKszs5ypKOglzZa0UtIqSZd3s/0CSY9IWirpbknTyrZ9ofi6lZI+WM3i+7xjj4Vf/AKWLoV3vxu+8pUU+N/4Rhrb3sysCvYa9JIagPnAGcA04JzyIC+6LiKOi4gTgW8B3y6+dhpwNvB2YDbwveL7WbkTToBf/Qra2uDUU+GLX0wXba+8Et54I+vqzKyPq+SMfgawKiLaI2ILsAA4q3yHiChvbxgGlMbpPQtYEBGbI+JJYFXx/aw7J58MN90E998PLS3w+c+nwP+3f4NNm7Kuzsz6qEqCfizwbNnymuK6TiRdKGk16Yz+on187fmS2iS1dXR0VFp7fs2YAbfcAvfcA8cfD5dckgL/u9+Ft97Kujoz62OqdjE2IuZHxFHAZcCX9/G1V0dES0S0NDY2Vqukvm/WLLj9dvjjH2HqVLjoonSn7fe/D5s3Z12dmfURlQT9WmB82fK44rrdWQB8eD9fa9057TS44w74wx9S//t//MfUVfNHP4KtW7OuzsxqXCVBvxiYJKlZ0kDSxdWF5TtImlS2+CHgieLzhcDZkgZJagYmAQ8ceNl16t3vhj/9CW67DY44As4/PwX+tdfCtm1ZV2dmNWqvQR8R24C5wK3ACuCGiFguaZ6kOcXd5kpaLmkpcAnwqeJrlwM3AI8BvwMujIjtPfA56ocE738/LFoEN98Mo0alAdSmTk03YznwzawLRY1NZN3S0hJtbW1Zl9F3RMBvfpPurn3oIZgyJT3/+7+HBvdkNasXkpZEREt323xnbF8npTHwlyxJffEHDYKPfSyNh3/dde6WaWYO+tyQ4MMfTmf1v/hFWv74x2H06PTz179210yzOuWgz5t+/eBv/zaNo3P77ens/tZb00Fg9Og0Pv5NN7l7plkdcdDnVUMDvO998MMfptmtbr01tdv/9rcwZw4cdhice266oLtlS9bVmlkPctDXgwED4AMfgGuuSTNc3XJLGhf/17+GD30ohf6nPw2/+5375ZvlkIO+3gwYALNnp773L7yQeuzMmQO//CWccQYcfjh85jOp2cddNc1ywUFfzwYOTGf0P/kJrF8PCxfCX/0V3HBD+gYwZgz8wz/A73/v0Dfrwxz0lgwalLpp/vSnKfR/9at0Y9bPfpba+o84Aj77WbjzTtjue97M+hIHve1q8ODUS+e666CjIzXrvOc98J//mYZhGDsW5s5NwzE49M1qnoPe9mzIkHThdsGCdKZ/ww1QKKQ2/tNPh/Hj06iad98NO3ZkXa2ZdcNBb5UbNgz+7u/SDVnr16fwP+WUNIpmoQATJsDnPgf33uvQN6shDnrbP8OHw0c/mpp11q9PbfktLfC976Vx9Jua4NJL02xZNTaeklm98aBmVl2vvJLuvL3hhp398seMSXPhnnpqOghMn566eZpZ1expUDMHvfWcjRvTTVm33ZamRXz66bR+yJA0XeKsWSn8TzkFRo7MtlazPs5Bb7Vh7do0jv6iRSn4H3poZ//8Y47pfNY/aVIamM3MKuKgt9r05puweHEK/XvuSQeAjRvTtsbGFPils/6TT07dPs2sW3sK+v69XYzZXwwdmrponn56Wt6xAx5/fGfo33NPavqBdBfvySd3PusfPTq72s36EJ/RW21bvz511yyd9be17Rxt8+ijd4b+qaem5p9+7khm9clNN5Yfmzen2bRKZ/z33JPu3gV429vShd1S+M+Ykfr+m9UBB73lVwSsXt25nX/58rStoSF15Sxv529q8ly6lksOeqsvL7+cmntKZ/33379z7tyBA1OPnqlT00TqU6fufH7wwdnWbXYAHPRW37ZuhYcfhkceSRd7H38cVq6EVas6D8o2Zsyu4T91ahrawW3/VuPc68bq24ABaXiGli7/B7Zsgfb2FPql8H/8cfj5z9O3gpLBg2Hy5M7hP2VKegwf3rufxWw/VHRGL2k28B2gAbgmIq7osv0S4DPANqAD+HREPF3cth14pLjrMxExZ0+/y2f0lrkIePHFzuFfet7e3nnAtnHjOod/6dvA2LH+FmC96oCabiQ1AH8G3g+sARYD50TEY2X7vBu4PyLelPRZ4F0R8dHittcjouLTHge91bTNm1OTT9dvAY8/Dq++unO/oUN3PQBMmZK+GQwdml39llsH2nQzA1gVEe3FN1sAnAX8Jegj4o6y/e8DPrH/5ZrVsEGD4O1vT49yEWkO3q7hf999aTjn8hOqsWOhuRkmTtz155gx/iZgVVdJ0I8Fni1bXgO8cw/7nwfcUrY8WFIbqVnnioj4v11fIOl84HyACRMmVFCSWY2R0sTqhx8O73pX522bNqVvAaXwb29PjzvuSFM3lh8EBg1KXUC7Owg0N8Mhh/Tmp7KcqOrFWEmfAFqA08tWHxkRayVNBP4g6ZGIWF3+uoi4GrgaUtNNNWsyy9yQIXDccenR1ebN8MwzKfiffLLzz3vv3Tn2T8nIkbs/CEyYkLqPmnVRSdCvBcaXLY8rrutE0vuALwGnR8Tm0vqIWFv82S7pTmA6sLrr683q0qBBqV//pEndb3/55RT8XQ8CDz+cxgEqDQcBqcln3LgU/N0dDEaP9oigdaqSoF8MTJLUTAr4s4GPle8gaTrwQ2B2RKwvWz8CeDMiNksaBZwKfKtaxZvl3ogR6XHSSbtu274dnntu14PAk0/CLbfAunWd9x86NIV+6TF2bGpqGjNm589DD/U1ghzaa9BHxDZJc4FbSd0rr42I5ZLmAW0RsRC4EhgO/ELpjKHUjfIY4IeSdpCmLbyivLeOmR2AhoY0Ofv48XDaabtu37QJnnqq+wPBn/7UuZdQSf/+cNhhux4AStcfyp8PGdLjH9Gqw3fGmtWrN96A559Pj3Xr0qP0vPzn+vXdT/Z+yCG7Hgy6OzgceqibjHqB74w1s10NGwZHHZUee7J9exohdHcHgnXr4IEH0s8339z19QMGpG8JuzsYlL5BHHaY7zHoIQ56M9uzhoadZ+d7EgGvv77nA8LTT6d7C158sXO30pLhwzsHf+nRdfmwwzwE9T5w0JtZdUhw0EHpMXnynvfdujU1CT3/fLrR7IUXOj8v3Xx2552wYUP371E6KOzuYFC+XOcHBQe9mfW+AQNSr5+xY/e+75Ytqemou4NBad3KlekC80svdf8ew4bt+UAwenS6ljByZOrlNGBAdT9vxhz0ZlbbBg6s/KCwdWs6KOzugPDCC/DnP8Ndd6Xmo905+OAU+qVH6SCwp+UaPkA46M0sPwYMgCOOSI+9KR0UXnghNSO9/HL6RrBhw85HafnZZ3c+764HUslBB+16INjbQWLkyB4/QDjozaw+7ctBoWTHDnjttc4Hga7Pux4gSuvKJ7npqnSAOOUUuP76A/9sXTjozcwq1a9fun/gkEPS3cWV2t0BouvyuHE9UraD3sysp+3vAaJav77Xf6OZmfUqB72ZWc456M3Mcs5Bb2aWcw56M7Occ9CbmeWcg97MLOcc9GZmOVdzM0xJ6gCePoC3GAXsYbSiuuK/RWf+e3Tmv8dOefhbHBkRjd1tqLmgP1CS2nY3nVa98d+iM/89OvPfY6e8/y3cdGNmlnMOejOznMtj0F+ddQE1xH+Lzvz36Mx/j51y/bfIXRu9mZl1lsczejMzK+OgNzPLudwEvaTZklZKWiXp8qzryZKk8ZLukPSYpOWSLs66pqxJapD0kKTfZF1L1iS9TdKNkh6XtELSKVnXlCVJnyv+P3lU0vWSBmddU7XlIuglNQDzgTOAacA5kqZlW1WmtgGXRsQ0YCZwYZ3/PQAuBlZkXUSN+A7wu4iYCpxAHf9dJI0FLgJaIuJYoAE4O9uqqi8XQQ/MAFZFRHtEbAEWAGdlXFNmImJdRDxYfP4a6T/y2Gyryo6kccCHgGuyriVrkg4BTgP+D0BEbImIjdlWlbn+wBBJ/YGhwHMZ11N1eQn6scCzZctrqONgKyepCZgO3J9tJZm6Cvg8sCPrQmpAM9AB/LjYlHWNpGFZF5WViFgL/CvwDLAOeCUibsu2qurLS9BbNyQNB34J/HNEvJp1PVmQdCawPiKWZF1LjegPnAR8PyKmA28AdXtNS9II0rf/ZuAIYJikT2RbVfXlJejXAuPLlscV19UtSQNIIf+ziPjvrOvJ0KnAHElPkZr03iPpv7ItKVNrgDURUfqGdyMp+OvV+4AnI6IjIrYC/w3MyrimqstL0C8GJklqljSQdDFlYcY1ZUaSSG2wKyLi21nXk6WI+EJEjIuIJtK/iz9ERO7O2CoVEc8Dz0qaUlz1XuCxDEvK2jPATElDi/9v3ksOL073z7qAaoiIbZLmAreSrppfGxHLMy4rS6cCnwQekbS0uO6LEXFzhjVZ7fgn4GfFk6J24H9kXE9mIuJ+STcCD5J6qz1EDodD8BAIZmY5l5emGzMz2w0HvZlZzjnozcxyzkFvZpZzDnozs5xz0FtdkrRd0tKyR9XuDpXUJOnRar2f2YHKRT96s/2wKSJOzLoIs97gM3qzMpKekvQtSY9IekDS0cX1TZL+IGmZpN9LmlBcf5ikX0l6uPgo3T7fIOlHxXHOb5M0JLMPZXXPQW/1akiXppuPlm17JSKOA/6DNPIlwHeBn0TE8cDPgH8vrv934I8RcQJpzJjSHdmTgPkR8XZgI/CRHv48ZrvlO2OtLkl6PSKGd7P+KeA9EdFeHBju+Yg4VNKLwJiI2Fpcvy4iRknqAMZFxOay92gCbo+IScXly4ABEfEvPf/JzHblM3qzXcVunu+LzWXPt+PrYZYhB73Zrj5a9vPe4vNF7Jxi7uPAXcXnvwc+C3+Zl/aQ3irSrFI+y7B6NaRsZE9Ic6iWuliOkLSMdFZ+TnHdP5FmZfqfpBmaSiM+XgxcLek80pn7Z0kzFZnVDLfRm5UpttG3RMSLWddiVi1uujEzyzmf0ZuZ5ZzP6M3Mcs5Bb2aWcw56M7Occ9CbmeWcg97MLOf+PzurfsheXyh6AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEWCAYAAABollyxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAdvElEQVR4nO3deZBcdb338feHSYCEhCWbAZKQAEkkPAKBqRABDQWigI8LKAoIguUjz73ldn1wgedaijzX8hZYXriCC1cFBBSR65LLRdFSlm4IkoQlgUBiCBKyIBOyyZZtvs8fv9NOz2SS6Zn05PSc/ryqurr7nNPT3+7KfPKb7znndxQRmJlZce2RdwFmZta/HPRmZgXnoDczKzgHvZlZwTnozcwKzkFvZlZwDnozs4Jz0FuhSLpP0jpJe+Vdi1mjcNBbYUiaCLwNCOC9u/F9B+2u9zLrCwe9FclHgYeBm4CLKgsljZf0C0ltkl6WdF3Vuk9IelrS3yQtknRstjwkHV613U2S/iV7fLKkFZK+JOlF4EZJB0i6K3uPddnjcVWvHyHpRkmrsvW/ypY/Kek9VdsNlrRG0vR++5as6TjorUg+CtyW3d4l6U2SWoC7gOeBicDBwO0Aks4Brshety/pr4CXa3yvscAI4BDgEtLv0o3Z8wnA68B1VdvfAgwFjgTGAP+WLf8xcEHVdmcCqyPisRrrMOuRPNeNFYGkk4B7gQMjYo2kZ4Dvk0b4s7PlW7u85h7g7oi4tpufF8DkiFiaPb8JWBERX5Z0MvA7YN+IeGMH9RwD3BsRB0g6EFgJjIyIdV22OwhYDBwcERsl3Qk8EhFX9fnLMOvCI3oriouA30XEmuz5T7Jl44Hnu4Z8ZjzwbB/fr6065CUNlfR9Sc9L2gg8AOyf/UUxHljbNeQBImIV8CDwAUn7A2eQ/iIxqxvvRLIBT9IQ4ENAS9YzB9gL2B/4KzBB0qBuwv4F4LAd/NjXSK2WirHAiqrnXf8UvhSYChwfES9mI/rHAGXvM0LS/hGxvpv3uhn4X6TfxzkRsXLHn9as9zyityJ4P7ANmAYck92OAErZutXAv0raR9Lekk7MXvcD4POSjlNyuKRDsnWPA+dLapF0OjCrhxqGk/ry6yWNAL5aWRERq4HfAN/JdtoOlvT2qtf+CjgW+CypZ29WVw56K4KLgBsjYnlEvFi5kXaGnge8BzgcWE4alX8YICJ+Dnyd1Ob5GylwR2Q/87PZ69YDH8nW7cw1wBBgDWm/wG+7rL8Q2AI8A7wE/FNlRUS8DvwnMAn4RS8/u1mPvDPWrAFI+gowJSIu6HFjs15yj94sZ1mr5+OkUb9Z3bl1Y5YjSZ8g7az9TUQ8kHc9Vkxu3ZiZFZxH9GZmBddwPfpRo0bFxIkT8y7DzGxAmT9//pqIGN3duoYL+okTJzJv3ry8yzAzG1AkPb+jdW7dmJkVnIPezKzgHPRmZgXnoDczKzgHvZlZwTnozcwKzkFvZlZwDXccvZlZEW3dChs3woYNO76NHg2XXFL/93bQm5n1YPPmHYdzT+Fd2ebVV3t+n5kzHfRmZn0SkYL25ZfTbe3ajseV5zsL8De6vQR8Z0OHwn77db6NH7/9su5u++6b7vfaq38+v4PezAaUrVth3brOQd3Tbe1a2LRpxz9z2DDYf/+OwB01Cg47rOdgrn4+ePDu+w56y0FvZrmpHmXXelvf3eXVM4MGwciRHbfDD4fjj++8rOttxIjGDul6cNCbWV1FpBH06tXw4ovpfke3V17Z8c8ZPrxzIB922M4De+TI9Bpp933WgcJBb2Y12bYNXnpp58FdCffNm7d//T77wIEHptv06XDmmTB2bGqTdDfK3nPP3f8Zi8pBb9bkNm3qeeS9enUK+fb27V8/YkRHgE+Z0vH4wANTkFceDx+++z+bJQ56s4KLgJUr4Zln0m3xYliyJC1bvTq1WbraYw8YM6YjpI89tnOAVwd5fx0pYvXjoDcriDfegD//OQV5JdQrwV7dC9933zTynjIFZs3qfgQ+Zgy0tOT3Way+HPRmA0gErFnTOcgrt+eeS+srDjkE3vxmOPHEdF+5jR3rHZbNxkFv1oC2bEnB3V2gr1vXsd2QITB1KsyYARde2BHmkyennZ9m4KA3y9X69du3Wp55Bp59NoV9xYEHpgD/8Ic7j87Hj0/9dLOdcdCb9bNNm9LofOnSjh56JdxffLFju8GD00j8iCPgrLM6wnzq1HT2pVlfOejN6uC112DZshTmXW/Ll3funY8YkcL8zDM7j84nTUpndprVm/9ZmdXolVdSS6UyMq8O85UrO287alQ6/f5tb0v3ldthh6V1ZruTg96syoYN3Y/Kly7t3GYBeNObUni/4x3pfvLkjjDff/986jfrjoPemkplHpYdhfmaNZ23P/jgFN7vfvf2I3Of6WkDhYPeCu355+G+++D++2HhwhTm1bMfSunIlcMPh7PP3j7Mhw7NrXSzunHQW2FEwF/+0hHs992Xgh7SDtDjjoPzz+8c5pMmwd5751i02W7goLcBKyLtHK2E+v33wwsvpHWjRqXT+y+9FE4+GY480sebW/Ny0NuAEZGOdqkO9srRLmPGpGC/7LJ0P22aT/M3q3DQW8OKSCcWVUL9/vvTbIuQ5muZNSuN1mfNSsehO9jNuuegt4YRAU8/3TnY//rXtO6gg1KoV4J9yhQHu1mtHPSWm/Z2WLSoc7C3taV148bBaad1jNoPO8zBbtZXDnrbbdrb0yGOlR77Aw+kiz0DTJgAZ5zREeyTJjnYzerFQW/9aulSuOuuFOylUsfVjCZOhPe8pyPYJ07Mr0azonPQW9397W/w85/DjTdCuZyWHXoovP/9KdhnzUoXxTCz3cNBb3XR3p5G7DfeCHfeCa++mnaYfuMbcN55DnazPNUU9JJOB64FWoAfRMS/dll/CPAjYDSwFrggIlZIOgb4LrAvsA34ekT8rI71W86efx5uvjndli1L87+cfz587GMwc6b77GaNoMegl9QCXA+cBqwA5kqaHRGLqjb7JvDjiLhZ0inAN4ALgdeAj0bEnyUdBMyXdE9ErMcGrNdeg1/+Mo3e//jHdFjkKafA176W5ovx/DBmjaWWEf0MYGlELAOQdDvwPqA66KcB/yd7fC/wK4CIWFLZICJWSXqJNOp30A8wEfDwwyncf/Yz2LgxHRlzxRVw0UVuzZg1slqC/mDgharnK4Dju2zzBHA2qb1zFjBc0siIeLmygaQZwJ7As7tUse1Wq1bBLbfATTelS98NHQrnnAMXXwxvf7vnjzEbCOq1M/bzwHWSLgYeAFaSevIASDoQuAW4KCLau75Y0iXAJQATJkyoU0nWV5s2wezZafR+zz1pR+tJJ8EPf5hC3vOwmw0stQT9SmB81fNx2bK/i4hVpBE9koYBH6j04SXtC/w38M8R8XB3bxARNwA3ALS2tkZ321j/ioDHHkvh/pOfpOPdx42Dyy9Po/fDD8+7QjPrq1qCfi4wWdIkUsCfC5xfvYGkUcDabLR+OekIHCTtCfyStKP2znoWbvXR1ga33poCfuFC2GsvOOusdNTMqadCS0veFZrZruox6CNiq6RPAfeQDq/8UUQ8JelKYF5EzAZOBr4hKUitm09mL/8Q8HZgZNbWAbg4Ih6v78ew3tiyBX7zmxTud90FW7fCjBnw3e/Cuef6eqdmRaOIxuqUtLa2xrx58/Iuo5CefDKF+623wksvpYtbX3hhas0ceWTe1ZnZrpA0PyJau1vnM2MLbu1auP32FPDz5sHgwWmOmY99DE4/HQb5X4BZ4fnXvIAi0hwz110Hv/oVbN4MRx8N11wDH/lIusyemTUPB32BtLfDr38NV12VTm4aMQL+4R/S6P2YY/Kuzszy4qAvgE2b0klNV18NS5akM1avvz713j0dgZk56Aew9evhe9+Da6+FF1+EY49N0xOcfbZ772bWwXEwAK1cmfrt3/9+mvv9ne9MR9KccopnizSz7TnoB5BFi1J75rbbUj/+Qx+CL37R/Xcz2zkHfYOLgAcfTDtY/+u/YMiQtIP1c59LvXgzs5446BtUe3uaWOyqq2DOHBg5Mk0J/MlP+vBIM+sdB32D2bQp9duvvhoWL06j9uuuS4dI+ggaM+sLB32D2LCh4wia1ath+vR0RusHPuAjaMxs1zhCcrZyZQr3730vHUFz2mnw4x+nmSN9BI2Z1YODPidPP53aM7feCtu2pSNovvCFdCy8mVk9Oeh3s8oRNLNnpyNoLrkELr3UR9CYWf9x0O8G7e3p0MirroKHHkpz0Hz1q+kImtGj867OzIrOQd+PNm1KJzddfXW6sPbEifDtb6cjaPbZJ+/qzKxZOOj7wYYNaXqCa65JR9Acc0y6Dus55/gIGjPb/Rw7dbZsGbS2wrp16ciZm2+Gd7zDR9CYWX4c9HX261+nkC+V4KST8q7GzAz2yLuAoimV4NBDHfJm1jgc9HVUuYSfQ97MGomDvo6WLIG2Nnjb2/KuxMysg4O+jsrldO8RvZk1Egd9HZVKaQrhqVPzrsTMrIODvo4q/XkfSmlmjcRBXyerV8Ozz7o/b2aNx0FfJ+7Pm1mjctDXSamUrgA1fXrelZiZdeagr5NyGWbOhMGD867EzKwzB30dbNwITzzh/ryZNSYHfR3MmZPmnHd/3swakYO+DkolaGlJrRszs0bjoK+DcjnthB02LO9KzMy256DfRZs2wZ/+5P68mTWumoJe0umSFktaKumybtYfIukPkhZIuk/SuKp1F0n6c3a7qJ7FN4L58+GNN9yfN7PG1WPQS2oBrgfOAKYB50ma1mWzbwI/joijgCuBb2SvHQF8FTgemAF8VdIB9Ss/fz5RyswaXS0j+hnA0ohYFhGbgduB93XZZhrwx+zxvVXr3wX8PiLWRsQ64PfA6bteduMolWDKFBgzJu9KzMy6V0vQHwy8UPV8Rbas2hPA2dnjs4DhkkbW+NoBq70dHnzQ/Xkza2z12hn7eWCWpMeAWcBKYFutL5Z0iaR5kua1tbXVqaT+t2hRuj6s2zZm1shqCfqVwPiq5+OyZX8XEasi4uyImA78c7ZsfS2vzba9ISJaI6J19OjRvfwI+an05z2iN7NGVkvQzwUmS5okaU/gXGB29QaSRkmq/KzLgR9lj+8B3inpgGwn7DuzZYVQKsHYseli4GZmjarHoI+IrcCnSAH9NHBHRDwl6UpJ7802OxlYLGkJ8Cbg69lr1wL/j/SfxVzgymxZIZTLaTTvC42YWSNTRORdQyetra0xb968vMvo0fLlcMghcO218JnP5F2NmTU7SfMjorW7dT4zto/cnzezgcJB30elEgwfDkcdlXclZmY756Dvo3IZTjghzVppZtbIHPR9sHYtPPmkj583s4HBQd8HDz2U7t2fN7OBwEHfB6VSujbsjBl5V2Jm1jMHfR+Uy9DaCkOG5F2JmVnPHPS99PrrMHeu2zZmNnA46Htp7lzYssU7Ys1s4HDQ91KplO5PPDHfOszMauWg76VyGY48EkaMyLsSM7PaOOh7Ydu2dGil+/NmNpA46Hth4ULYuNH9eTMbWBz0vVDpz3tEb2YDiYO+F8plGD8eJkzIuxIzs9o56GsUkUb0Hs2b2UDjoK/Rc8/B6tXuz5vZwOOgr5H782Y2UDnoa1QuwwEHwLRpeVdiZtY7DvoalUrpbNg9/I2Z2QDj2KpBWxssXuz+vJkNTA76GvhC4GY2kDnoa1Auw157wXHH5V2JmVnvOehrUCrB8censDczG2gc9D149VV49FH3581s4HLQ9+Dhh9Osle7Pm9lA5aDvQbkMErz1rXlXYmbWNw76HpRKcPTRsN9+eVdiZtY3Dvqd2LIltW7cnzezgcxBvxOPP552xro/b2YDmYN+JyonSnlEb2YDmYN+J0olOPRQOOigvCsxM+s7B/0ORKQRvUfzZjbQOeh3YMmSNJmZ+/NmNtDVFPSSTpe0WNJSSZd1s36CpHslPSZpgaQzs+WDJd0saaGkpyVdXu8P0F/cnzezougx6CW1ANcDZwDTgPMkdb38xpeBOyJiOnAu8J1s+TnAXhHxFuA44H9Lmlif0vtXqQSjRsHUqXlXYma2a2oZ0c8AlkbEsojYDNwOvK/LNgHsmz3eD1hVtXwfSYOAIcBmYOMuV70blEppNC/lXYmZ2a6pJegPBl6oer4iW1btCuACSSuAu4FPZ8vvBF4FVgPLgW9GxNpdKXh3WLUKli1zf97MiqFeO2PPA26KiHHAmcAtkvYg/TWwDTgImARcKunQri+WdImkeZLmtbW11amkvnN/3syKpJagXwmMr3o+LltW7ePAHQARMQfYGxgFnA/8NiK2RMRLwINAa9c3iIgbIqI1IlpHjx7d+09RZ+UyDB0K06fnXYmZ2a6rJejnApMlTZK0J2ln6+wu2ywHTgWQdAQp6Nuy5adky/cBZgLP1Kf0/lMqwcyZMHhw3pWYme26HoM+IrYCnwLuAZ4mHV3zlKQrJb032+xS4BOSngB+ClwcEUE6WmeYpKdI/2HcGBEL+uOD1MuGDbBggfvzZlYcg2rZKCLuJu1krV72larHi4ATu3ndK6RDLAeMOXOgvd39eTMrDp8Z20W5DC0tqXVjZlYEDvouSqW0E3bYsLwrMTOrDwd9lU2b4JFH3J83s2Jx0FeZPx/eeMP9eTMrFgd9FZ8oZWZF5KCvUirBlCkwZkzelZiZ1Y+DPtPeDg8+6P68mRWPgz6zaBGsW+e2jZkVj4M+U+nPe0RvZkXjoM+USjB2bLoYuJlZkTjoM+VyGs37QiNmVjQOemD58nRzf97MishBj/vzZlZsDnpSf374cDjqqLwrMTOrPwc9aUR/wglp1kozs6Jp+qBfuxaefNL9eTMrrqYP+oceSvfuz5tZUTV90JdK6dqwM2bkXYmZWf9o+qAvl6G1FYYMybsSM7P+0dRB//rrMHeu+/NmVmxNHfRz58KWLe7Pm1mxNXXQl0rp/oQT8q3DzKw/NXXQl8tw5JEwcmTelZiZ9Z+mDfpt29Khle7Pm1nRNW3QL1wIGze6P29mxde0QV/pz3tEb2ZF17RBXy7D+PFwyCF5V2Jm1r+aMugj0ojebRszawZNGfTPPQerV7ttY2bNoSmDvtKf94jezJpBUwZ9uQwHHADTpuVdiZlZ/2vKoC+V4MQTYY+m/PRm1myaLura2mDxYvfnzax5NF3Q+0LgZtZsagp6SadLWixpqaTLulk/QdK9kh6TtEDSmVXrjpI0R9JTkhZK2rueH6C3ymXYay847rg8qzAz230G9bSBpBbgeuA0YAUwV9LsiFhUtdmXgTsi4ruSpgF3AxMlDQJuBS6MiCckjQS21P1T9EKpBMcfn8LezKwZ1DKinwEsjYhlEbEZuB14X5dtAtg3e7wfsCp7/E5gQUQ8ARARL0fEtl0vu29efRUefdT9eTNrLrUE/cHAC1XPV2TLql0BXCBpBWk0/+ls+RQgJN0j6VFJX9zFenfJww+nWSvdnzezZlKvnbHnATdFxDjgTOAWSXuQWkMnAR/J7s+SdGrXF0u6RNI8SfPa2trqVNL2SiWQ4K1v7be3MDNrOLUE/UpgfNXzcdmyah8H7gCIiDnA3sAo0uj/gYhYExGvkUb7x3Z9g4i4ISJaI6J19OjRvf8UNSqX4eijYb/9+u0tzMwaTi1BPxeYLGmSpD2Bc4HZXbZZDpwKIOkIUtC3AfcAb5E0NNsxOwtYRA62bIE5c9yfN7Pm0+NRNxGxVdKnSKHdAvwoIp6SdCUwLyJmA5cC/yHpc6QdsxdHRADrJH2L9J9FAHdHxH/314fZmccfh9dec3/ezJpPj0EPEBF3k9ou1cu+UvV4EXDiDl57K+kQy1z5QiNm1qya5szYchkOPRQOOijvSszMdq+mCPqIFPQezZtZM2qKoF+yJE1m5v68mTWjpgh69+fNrJk1RdCXyzBqFEydmnclZma7X1MEfamURvNS3pWYme1+hQ/6Vatg2TL3582seRU+6CsXGnF/3syaVVME/dChMH163pWYmeWj8EFfKsHMmTB4cN6VmJnlo9BBv2EDLFjg/ryZNbdCB/2cOdDe7v68mTW3Qgd9uQwtLal1Y2bWrAod9KVS2gk7bFjelZiZ5aewQb9pEzzyiPvzZmaFDfr58+GNN9yfNzMrbND7RCkzs6SwQV8qwZQpMGZM3pWYmeWrkEHf3g4PPuj+vJkZFDToFy2CdevctjEzg4IGfaU/7xG9mVlBg75UgrFj08XAzcyaXSGDvlxOo3lfaMTMrIBBv3x5urk/b2aWFC7o3Z83M+uscEFfKsHw4XDUUXlXYmbWGAoX9OUynHBCmrXSzMwKFvRr18KTT7o/b2ZWrVBB/9BD6d79eTOzDoUK+lIpXRt2xoy8KzEzaxyFCvpyGVpbYciQvCsxM2schQn611+HuXPdnzcz66owQb9hA3zwg/Cud+VdiZlZYxmUdwH1MnYs/OQneVdhZtZ4ahrRSzpd0mJJSyVd1s36CZLulfSYpAWSzuxm/SuSPl+vws3MrDY9Br2kFuB64AxgGnCepGldNvsycEdETAfOBb7TZf23gN/serlmZtZbtYzoZwBLI2JZRGwGbgfe12WbAPbNHu8HrKqskPR+4DngqV0v18zMequWoD8YeKHq+YpsWbUrgAskrQDuBj4NIGkY8CXga7tcqZmZ9Um9jro5D7gpIsYBZwK3SNqD9B/Av0XEKzt7saRLJM2TNK+tra1OJZmZGdR21M1KYHzV83HZsmofB04HiIg5kvYGRgHHAx+UdBWwP9Au6Y2IuK76xRFxA3ADQGtra/Tlg5iZWfdqCfq5wGRJk0gBfy5wfpdtlgOnAjdJOgLYG2iLiL/POiPpCuCVriFvZmb9q8fWTURsBT4F3AM8TTq65ilJV0p6b7bZpcAnJD0B/BS4OCI8MjczawBqtDyW1AY8vws/YhSwpk7lDHT+Ljrz99GZv48ORfguDomI0d2taLig31WS5kVEa951NAJ/F535++jM30eHon8XhZnrxszMuuegNzMruCIG/Q15F9BA/F105u+jM38fHQr9XRSuR29mZp0VcURvZmZVHPRmZgVXmKDvac78ZiJpfHZ9gEWSnpL02bxrypuklux6CXflXUveJO0v6U5Jz0h6WtJb864pT5I+l/2ePCnpp9kULoVSiKCvcc78ZrIVuDQipgEzgU82+fcB8FnSmd0G1wK/jYg3A0fTxN+LpIOBzwCtEfE/gBbSNC+FUoigp7Y585tGRKyOiEezx38j/SJ3nVq6aUgaB7wb+EHeteRN0n7A24EfAkTE5ohYn29VuRsEDJE0CBhK1fU0iqIoQV/LnPlNSdJEYDrwp3wrydU1wBeB9rwLaQCTgDbgxqyV9QNJ++RdVF4iYiXwTdLEjKuBDRHxu3yrqr+iBL11I7vwy38C/xQRG/OuJw+S/ifwUkTMz7uWBjEIOBb4bnbpz1eBpt2nJekA0l//k4CDgH0kXZBvVfVXlKCvZc78piJpMCnkb4uIX+RdT45OBN4r6S+klt4pkm7Nt6RcrQBWRETlL7w7ScHfrN4BPBcRbRGxBfgFcELONdVdUYL+73PmS9qTtDNlds415UaSSD3YpyPiW3nXk6eIuDwixkXERNK/iz9GROFGbLWKiBeBFyRNzRadCizKsaS8LQdmShqa/d6cSgF3Ttdy4ZGGFxFbJVXmzG8BfhQRzXwx8hOBC4GFkh7Plv3fiLg7x5qscXwauC0bFC0DPpZzPbmJiD9JuhN4lHS02mMUcDoET4FgZlZwRWndmJnZDjjozcwKzkFvZlZwDnozs4Jz0JuZFZyD3pqSpG2SHq+61e3sUEkTJT1Zr59ntqsKcRy9WR+8HhHH5F2E2e7gEb1ZFUl/kXSVpIWSHpF0eLZ8oqQ/Slog6Q+SJmTL3yTpl5KeyG6V0+dbJP1HNs/57yQNye1DWdNz0FuzGtKldfPhqnUbIuItwHWkmS8Bvg3cHBFHAbcB/54t/3fg/og4mjRnTOWM7MnA9RFxJLAe+EA/fx6zHfKZsdaUJL0SEcO6Wf4X4JSIWJZNDPdiRIyUtAY4MCK2ZMtXR8QoSW3AuIjYVPUzJgK/j4jJ2fMvAYMj4l/6/5OZbc8jerPtxQ4e98amqsfb8P4wy5GD3mx7H666n5M9foiOS8x9BChlj/8A/CP8/bq0++2uIs1q5VGGNashVTN7QrqGauUQywMkLSCNys/Lln2adFWmL5Cu0FSZ8fGzwA2SPk4auf8j6UpFZg3DPXqzKlmPvjUi1uRdi1m9uHVjZlZwHtGbmRWcR/RmZgXnoDczKzgHvZlZwTnozcwKzkFvZlZw/x9qc10rQz+TagAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RKYqwyTWSgd7",
        "colab_type": "text"
      },
      "source": [
        "# Evaluate the Keras' Model Performance\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xT6ZDhqRxUCj",
        "colab_type": "code",
        "outputId": "5146b331-0433-42ce-e3c4-6403e2a59628",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        }
      },
      "source": [
        "keras_test_loss, keras_test_acc = model.evaluate(x_test, y_test)\n",
        "\n",
        "print('Test loss:', keras_test_loss)\n",
        "print('Test accuracy:', keras_test_acc)"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "313/313 [==============================] - 0s 1ms/step - loss: 0.2376 - accuracy: 0.9352\n",
            "Test loss: 0.23761823773384094\n",
            "Test accuracy: 0.9351999759674072\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5723ZvJV1Iy4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# if the model is just created, them I want to update saved model\n",
        "model.save('./mnist-model.h5') "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aAWzx8T8m3sC",
        "colab_type": "code",
        "outputId": "66179008-93c4-416d-f9fb-9ddf2ae382d3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 281
        }
      },
      "source": [
        "# get a random image index from the test set\n",
        "image_index = int(np.random.randint(0, x_test.shape[0], size=1)[0])\n",
        "expected_label = y_test[image_index]\n",
        "digit_image = x_test[image_index]\n",
        "# and plot it\n",
        "plt.title('Example %d. Label: %d' % (image_index, expected_label))\n",
        "plt.imshow(digit_image, cmap='Greys')\n",
        "plt.show()"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAEICAYAAACZA4KlAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAUBElEQVR4nO3deZCcdZ3H8feHkAACucwQA8xmIAvLAutGGYFCpBAXC9iiCBZXFrIRkVgLLljLsRZKQZUUguBF7S4auYIiqBwVwiKCYSFLIcqESkiQm51UyEEmRkgC4Uq++8fzDHaG7qcnfUx38vu8qrqm+/k+x7efmU8/Tz9P9zyKCMxs27ddqxsws6HhsJslwmE3S4TDbpYIh90sEQ67WSIc9q2IpC9KeqzVfbQDSbdIumKop92aOew5Sb2SNkhaX3L7j1b31SiSnhnw3N6XNKekPlnSfElv5T8nl9RGS5olaVV+u3zAvA+T9AdJ6yQ9LenwLejrEUlfbsiTbBJJ/yrp/yStldSzJc+vnTjsmzs+InYpuX211Q01SkQc0P+8gF2BpcCvACSNAGYDPwPGALOA2flwgO8DHwG6gIOBaZLOzKcdC8wBrgFGA98B5kgaM0RPrakkHQJcBZwEjAJuBO6RNKyljdXAYR8ESddLuqvk8dWS5iozRtJ9kvok/Tm/v2fJuI9IukLS4/kWdY6kj0q6Ld9SPCmpq2T8kHSepFckrZZ0jaSyvydJ+0l6SNIaSc9LOmWQT+kIYBzQ/5yOBLYHfhAR70TEdYCAo/L68cB3IuKtiOgl+4P/Ul47DFgZEb+KiI0R8TOgD/jCIHupSNKvJK2U9IakeZIOGDDKuPz5r5P0qKSJJdPWum4G6gKeiYj5kX3c9FaydbdbjfNrGYd9cC4A/i5/z/wZ4Cxgev7L3w64GZgI/BWwARi4+38aMA3YA5gE/C6fZizwLHDZgPFPBLqBTwIn8JdgfUDSzsBDwM/J/vBOA/5L0v6DeD7Tgbsi4s388QHA07H5Z6efzod/sMgB9w+sUCtXr9WvgX3Int9TwG0D6qcD3yIL34L++pauG0mvF+ya/xoYJumQfGv+pXxZK+t4Xq0REb5lf+O9wHrg9ZLb2SX1Q4A1wBJgasF8JgN/Lnn8CPCNksffBX5d8vh4YEHJ4wCOKXl8DjA3v/9F4LH8/qnA/w5Y9o+By6o8z48Aa4EjS4ZdCtwxYLzbgMvz+z8D7ibb/f9r4GXgnbz20XxdTQWGk72QbAJ+PMj1/gjw5UGMNzpfN6Pyx7eU9gzsAmwEOqutm3zaKwbZn4BLgPeA94HVwKda/fday81b9s1NiYjRJbef9Bci4vfAK2S//F/2D5f0EUk/lrRE0lpgHjB6wHu610rubyjzeJcBfSwtub8E2L1MrxOBQ/Kt0uuSXifb0n2synP8AtmL1qMlw9YDIweMNxJYl98/L+/zRbL39rcDrwJExJ/I9j7+LX9exwC/7a/XStIwSVdJejlfr715aVzJaB+sp4hYnz+v3al93ZRzFnAm2V7OCOAM4D5J5X4nbc1hHyRJ5wI7AMuBi0tKFwB/AxwSESPJ3g/Dh3dtt0Rnyf2/ypc50FLg0QEvTrtExL9Umfd04NbIN1u5Z4CPSyrt+eP5cCJiTUScHhEfi4gDyP5u/tA/YkQ8GhGfioixZG9X9iut1+ifyF5E/oHswFhXPry0xw/Wk6RdyN4WLaf2dVPOZOC+iHghIjZFxAPACrJjFVsVh30QJO0LXEH2qj4NuLjk1NSuZFu91/Mj0wPff9fiovzAXydwPvCLMuPcB+wraZqk4fntU5L+tuB57Al8luxoe6lHyHaBz5O0g6T+sxAP59NNyg8qDpN0LDCDbH30z/cT+fJHAtcCSyPiN1vwfLeXtGPJbTjZen0H+BPZW48ry0x3nKTD87MG3wKeiIiltaybAk8C/yhp7/yA7NHAvsDiGubVUg775uZo83PR90januw969URsTAiXiR7D/dTSTsAPwB2Insv9wTwQAP6mA3MJzsQ9N9kR783ExHrgM+THXxaTnbA6GqyvY9KpgG/i4iXB8zrXWAK8M9k77+/RPaW5t18lIOARWS79d8GTo+IZ0pmcTHZ818KTCA7wAiApM9IWl/l+V5P9oLZf7uZ7Kj3EmAZ8EeydTvQz8leXNfkPZ6RP58tWjf57/ozFXq7FbiD7AVxLXAd8JWIeK7Kc2o72nxvzlpNUgD7RMRLre7Fti3espslwmE3S4R3480S4S27WSK2H8qFjRs3Lrq6uoZykWZJ6e3tZfXq1WU/41FX2CUdA/wQGAbcEBFXFY3f1dVFT09PPYs0swLd3d0VazXvxucfB/1P4Fhgf2DqIL+EYWYtUM979oOBlyLilfzDF3eQfbzRzNpQPWHfg82/sPFqPmwzkmYo++8ePX19fXUszszq0fSj8RExMyK6I6K7o6Oj2YszswrqCfsyNv921p75MDNrQ/WE/UlgH0l75d86Og24tzFtmVmj1XzqLSLez78K+RuyU283DfgmlJm1kbrOs0fE/cD9DerFzJrIH5c1S4TDbpYIh90sEQ67WSIcdrNEOOxmiXDYzRLhsJslwmE3S4TDbpYIh90sEQ67WSIcdrNEOOxmiXDYzRLhsJslwmE3S4TDbpYIh90sEQ67WSIcdrNEOOxmiXDYzRLhsJslwmE3S4TDbpYIh90sEQ67WSIcdrNE1HUVV7NqNmzYULH29ttv1zXvkSNHFtZ7enoq1g477LDCaRcuXFhYP/DAAwvr7aiusEvqBdYBG4H3I6K7EU2ZWeM1Ysv+2YhY3YD5mFkT+T27WSLqDXsAD0qaL2lGuREkzZDUI6mnr6+vzsWZWa3qDfvhEfFJ4FjgXElHDBwhImZGRHdEdHd0dNS5ODOrVV1hj4hl+c9VwD3AwY1oyswar+awS9pZ0q7994HPA4sb1ZiZNVY9R+PHA/dI6p/PzyPigYZ0ZQ2zadOmwvoTTzxRWL/hhhvqWn7R/F944YW65n3SSScV1u+8886KtfzvtqJRo0bV1FM7qznsEfEK8PcN7MXMmsin3swS4bCbJcJhN0uEw26WCIfdLBH+ius2oOirnHfddVfhtNVOMc2bN6+w3tvbW1iPiIq1aqe/qik6tVavJUuWFNY7Ozubtuxm8ZbdLBEOu1kiHHazRDjsZolw2M0S4bCbJcJhN0uEz7NvBS688MLC+nXXXVexVu0rrtOmTSusT5w4sbB+6KGHFtaPOuqoirVqX3G99tprC+vVjBgxomJt8eLif72w11571bXsduQtu1kiHHazRDjsZolw2M0S4bCbJcJhN0uEw26WCJ9nbwPVLl08Z86cwnq1c+lFql16+Oabb6553gDLly+vWHvggfr+8/gVV1xRWD/++OMr1vbee++6lr018pbdLBEOu1kiHHazRDjsZolw2M0S4bCbJcJhN0uEz7O3geHDhxfWDzrooML6yy+/XPOyb7rppsL6brvtVlg/9dRTC+sXX3xxxdrdd99dOO3pp59eWD/ttNMK611dXYX11FTdsku6SdIqSYtLho2V9JCkF/OfY5rbppnVazC78bcAxwwY9nVgbkTsA8zNH5tZG6sa9oiYB6wZMPgEYFZ+fxYwpcF9mVmD1XqAbnxErMjvrwTGVxpR0gxJPZJ6+vr6alycmdWr7qPxkV25r+LV+yJiZkR0R0R3R0dHvYszsxrVGvbXJE0AyH+ualxLZtYMtYb9XmB6fn86MLsx7ZhZs1Q9zy7pduBIYJykV4HLgKuAX0o6C1gCnNLMJrd1w4YNK6yfeOKJhfW33nqrYu3BBx8snPb5558vrJ955pmF9aLz6ACrVlXe6fv2t79dOG21/5e/3Xb+TNiWqBr2iJhaofS5BvdiZk3kl0azRDjsZolw2M0S4bCbJcJhN0uEv+K6FTj55JNrri9YsKBw2u7u7pp66lftI9BFl02eMqX4KxU+tdZYXptmiXDYzRLhsJslwmE3S4TDbpYIh90sEQ67WSJ8nn0b19nZWVifNGlSYb2ef1MNxV/fXblyZeG0++67b13Lts15y26WCIfdLBEOu1kiHHazRDjsZolw2M0S4bCbJcLn2bdxb7zxRmG92Zfk2rBhQ8Xa+eefXzjt448/XljfaaedauopVd6ymyXCYTdLhMNulgiH3SwRDrtZIhx2s0Q47GaJ8Hn2bdw3v/nNwnq18/DVjB49urC+Zs2airV169YVTrtx48aaerLyqm7ZJd0kaZWkxSXDLpe0TNKC/HZcc9s0s3oNZjf+FuCYMsO/HxGT89v9jW3LzBqtatgjYh5QeV/MzLYK9Ryg+6qkp/Pd/DGVRpI0Q1KPpJ5mfw7bzCqrNezXA5OAycAK4LuVRoyImRHRHRHdHR0dNS7OzOpVU9gj4rWI2BgRm4CfAAc3ti0za7Sawi5pQsnDE4HFlcY1s/ZQ9Ty7pNuBI4Fxkl4FLgOOlDQZCKAX+EoTe7Qqrrnmmoq12bNnF04rqbA+Y8aMwnq167ufffbZFWu777574bTDhw8vrNuWqRr2iJhaZvCNTejFzJrIH5c1S4TDbpYIh90sEQ67WSIcdrNE+Cuu24CHH364Yu2dd96pa97nnXdeYX3ixImF9aJTb4sWLSqc9s033yys77DDDoV125y37GaJcNjNEuGwmyXCYTdLhMNulgiH3SwRDrtZInyefStw4YUXFtbnzp1bsTZixIjCaRcuXFhYnzRpUmH96KOPLqwXOeKIIwrro0aNqnne9mHespslwmE3S4TDbpYIh90sEQ67WSIcdrNEOOxmifB59jZQ7dLEy5cvL6xv2rSpYq3a983HjRtXWH/33XcL62PHji2sF1m6dGlhfe3atYX1MWMqXnXMyvCW3SwRDrtZIhx2s0Q47GaJcNjNEuGwmyXCYTdLxGAu2dwJ3AqMJ7tE88yI+KGkscAvgC6yyzafEhF/bl6r26733nuvsD5//vya533OOecU1qv97/WpU8tdxPcv5syZU1jfddddK9aqXU7a59EbazBb9veBCyJif+BQ4FxJ+wNfB+ZGxD7A3PyxmbWpqmGPiBUR8VR+fx3wLLAHcAIwKx9tFjClWU2aWf226D27pC7gE8DvgfERsSIvrSTbzTezNjXosEvaBbgL+FpEbPah5YgIsvfz5aabIalHUk9fX19dzZpZ7QYVdknDyYJ+W0TcnQ9+TdKEvD4BWFVu2oiYGRHdEdHd0dHRiJ7NrAZVwy5JwI3AsxHxvZLSvcD0/P50oPjQqpm11GC+4vppYBqwSNKCfNglwFXALyWdBSwBTmlOi9u+5557rrC+bNmymud96aWXFtavvvrqwvqaNWtqXjbAGWecUbHW2dlZ17xty1QNe0Q8BqhC+XONbcfMmsWfoDNLhMNulgiH3SwRDrtZIhx2s0Q47GaJ8L+SbgM77rhjYf3tt9+ued7r16+vq16vk08+uanzt8Hzlt0sEQ67WSIcdrNEOOxmiXDYzRLhsJslwmE3S4TPs7eBVn6ve9SoUYX1iy66qLA+ZUrx/xndb7/9trgnaw5v2c0S4bCbJcJhN0uEw26WCIfdLBEOu1kiHHazRPg8exvYaaedCuurV68urF955ZUVaz/60Y8Kp73zzjsL6wcddFBhfeTIkYV1ax/espslwmE3S4TDbpYIh90sEQ67WSIcdrNEOOxmiVBEFI8gdQK3AuOBAGZGxA8lXQ6cDfTlo14SEfcXzau7uzt6enrqbtrMyuvu7qanp6fsJdYH86Ga94ELIuIpSbsC8yU9lNe+HxHXNqpRM2ueqmGPiBXAivz+OknPAns0uzEza6wtes8uqQv4BPD7fNBXJT0t6SZJYypMM0NSj6Sevr6+cqOY2RAYdNgl7QLcBXwtItYC1wOTgMlkW/7vlpsuImZGRHdEdHd0dDSgZTOrxaDCLmk4WdBvi4i7ASLitYjYGBGbgJ8ABzevTTOrV9WwSxJwI/BsRHyvZPiEktFOBBY3vj0za5TBHI3/NDANWCRpQT7sEmCqpMlkp+N6ga80pUMza4jBHI1/DCh33q7wnLqZtRd/gs4sEQ67WSIcdrNEOOxmiXDYzRLhsJslwmE3S4TDbpYIh90sEQ67WSIcdrNEOOxmiXDYzRLhsJslouq/km7owqQ+YEnJoHFA8fWIW6dde2vXvsC91aqRvU2MiLL//21Iw/6hhUs9EdHdsgYKtGtv7doXuLdaDVVv3o03S4TDbpaIVod9ZouXX6Rde2vXvsC91WpIemvpe3YzGzqt3rKb2RBx2M0S0ZKwSzpG0vOSXpL09Vb0UImkXkmLJC2Q1NLrS+fX0FslaXHJsLGSHpL0Yv6z7DX2WtTb5ZKW5etugaTjWtRbp6T/kfRHSc9IOj8f3tJ1V9DXkKy3IX/PLmkY8AJwNPAq8CQwNSL+OKSNVCCpF+iOiJZ/AEPSEcB64NaIODAf9h1gTURclb9QjomIf2+T3i4H1rf6Mt751YomlF5mHJgCfJEWrruCvk5hCNZbK7bsBwMvRcQrEfEucAdwQgv6aHsRMQ9YM2DwCcCs/P4ssj+WIVeht7YQESsi4qn8/jqg/zLjLV13BX0NiVaEfQ9gacnjV2mv670H8KCk+ZJmtLqZMsZHxIr8/kpgfCubKaPqZbyH0oDLjLfNuqvl8uf18gG6Dzs8Ij4JHAucm++utqXI3oO107nTQV3Ge6iUucz4B1q57mq9/Hm9WhH2ZUBnyeM982FtISKW5T9XAffQfpeifq3/Crr5z1Ut7ucD7XQZ73KXGacN1l0rL3/eirA/CewjaS9JI4DTgHtb0MeHSNo5P3CCpJ2Bz9N+l6K+F5ie358OzG5hL5tpl8t4V7rMOC1edy2//HlEDPkNOI7siPzLwDda0UOFvvYGFua3Z1rdG3A72W7de2THNs4CPgrMBV4EfguMbaPefgosAp4mC9aEFvV2ONku+tPAgvx2XKvXXUFfQ7Le/HFZs0T4AJ1ZIhx2s0Q47GaJcNjNEuGwmyXCYTdLhMNuloj/B17wVOyrC2ORAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "50FQQAwQhQYN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# uncomment the following two lines to save sample images with digits\n",
        "#pil_img = tf.keras.preprocessing.image.array_to_img(digit_image.reshape((imgSize0,imgSize1,1)))\n",
        "#pil_img.save('./str(expected_label)+'.bmp')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LSOMagvap7W5",
        "colab_type": "code",
        "outputId": "7d601b1d-3c05-4793-88b5-d8c8dde8c463",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 104
        }
      },
      "source": [
        "# reshape the image for inference/prediction\n",
        "digit_image = digit_image.reshape(1, imgSize0, imgSize1, 1)\n",
        "# repeat few times to take the average execution time\n",
        "loop_count = 10\n",
        "\n",
        "start_time = time.time()\n",
        "for i in range(loop_count):\n",
        "    prediction = model.predict(digit_image)\n",
        "print(\"Keras inferences with %s second in average\" %((time.time() - start_time) / loop_count))\n",
        "\n",
        "print(prediction)\n",
        "predicted_label = prediction.argmax()\n",
        "print('Predicted value:', predicted_label)\n",
        "if (expected_label == predicted_label):\n",
        "  print('Correct prediction !')\n",
        "else:\n",
        "  print('Wrong prediction !')\n",
        "\n",
        "# one suggestion is to place this and the previous cell inside a for loop to perform several inferences"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Keras inferences with 0.032799935340881346 second in average\n",
            "[[5.1793786e-06 1.3430116e-03 1.7588119e-03 2.7343312e-03 7.1471358e-09\n",
            "  5.0697732e-03 1.4563107e-06 2.6127812e-04 9.8847544e-01 3.5068262e-04]]\n",
            "Predicted value: 8\n",
            "Correct prediction !\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TNJQ2AtWhGtI",
        "colab_type": "text"
      },
      "source": [
        "# Conversion from Keras to ONNX\n",
        "\n",
        "Now we know that the keras model is working, let's **convert it to ONNX** format and reevaluate the model again."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jmu8CmQBEs31",
        "colab_type": "code",
        "outputId": "42234039-6e46-4901-f66b-7a81a913395b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 52
        }
      },
      "source": [
        "# let's install the onnx packages from the source\n",
        "!pip install --quiet -U onnxruntime\n",
        "!pip install --quiet -U git+https://github.com/microsoft/onnxconverter-common\n",
        "!pip install --quiet -U git+https://github.com/onnx/keras-onnx"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  Building wheel for onnxconverter-common (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for keras2onnx (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SR5_5qAoFW8d",
        "colab_type": "text"
      },
      "source": [
        "Now we use Keras2onnx to convert the model to ONNX format and save it."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V3t9FZP1hERO",
        "colab_type": "code",
        "outputId": "54da3de0-07a7-4938-d2c9-5f93702cae84",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 781
        }
      },
      "source": [
        "import keras2onnx\n",
        "print(\"keras2onnx version is \"+keras2onnx.__version__)\n",
        "# convert to onnx model\n",
        "onnx_model = keras2onnx.convert_keras(model, 'mnist-onnx', debug_mode=1)\n",
        "output_model_path = \"./mnist-model.onnx\"\n",
        "# and save the model in ONNX format\n",
        "keras2onnx.save_model(onnx_model, output_model_path)"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tf executing eager_mode: True\n",
            "tf.keras model eager_mode: False\n",
            "Processing a keras layer - (dense_1: <class 'tensorflow.python.keras.layers.core.Dense'>)\n",
            "\toutput: dense_1/Identity:0\n",
            "\tinput : dense/Identity:0\n",
            "Processing a keras layer - (dense: <class 'tensorflow.python.keras.layers.core.Dense'>)\n",
            "\toutput: dense/Identity:0\n",
            "\tinput : flatten/Identity:0\n",
            "Processing a keras layer - (flatten: <class 'tensorflow.python.keras.layers.core.Flatten'>)\n",
            "\toutput: flatten/Identity:0\n",
            "\tinput : flatten_input:0\n",
            "var: flatten_input\n",
            "var: flatten_input:0\n",
            "var: flatten_input:01\n",
            "var: flatten/Identity:0\n",
            "var: dense/Identity:0\n",
            "var: dense_1/Identity:01\n",
            "var: dense_1/Identity:0\n",
            "var: dense_1\n",
            "Converting the operator (Identity): Identity\n",
            "Converting the operator (Identity1): Identity\n",
            "Converting the operator (Identity2): Identity\n",
            "Converting the operator (dense_1): <class 'tensorflow.python.keras.layers.core.Dense'>\n",
            "Converting the operator (dense): <class 'tensorflow.python.keras.layers.core.Dense'>\n",
            "Converting the operator (flatten): <class 'tensorflow.python.keras.layers.core.Flatten'>\n",
            "Converting the operator (flatten/Const): Const\n",
            "Converting the operator (Identity3): Identity\n",
            "The maximum opset needed by this model is only 11.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "keras2onnx version is 1.7.0\n",
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "flatten (Flatten)            (None, 784)               0         \n",
            "_________________________________________________________________\n",
            "dense (Dense)                (None, 8)                 6280      \n",
            "_________________________________________________________________\n",
            "dense_1 (Dense)              (None, 10)                90        \n",
            "=================================================================\n",
            "Total params: 6,370\n",
            "Trainable params: 6,370\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "None\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tmMT0GILGT4w",
        "colab_type": "text"
      },
      "source": [
        "# Evaluate the ONNX's Model Performance\n",
        "\n",
        "Now let's reevaluate the model read from the ONNX format to see if it is working."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q_ntfvpoGStw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import onnxruntime\n",
        "\n",
        "sess_options = onnxruntime.SessionOptions()\n",
        "sess = onnxruntime.InferenceSession(output_model_path, sess_options)\n",
        "data = [digit_image.astype(np.float32)]\n",
        "input_names = sess.get_inputs()\n",
        "feed = dict([(input.name, data[n]) for n, input in enumerate(sess.get_inputs())])"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VMqgCu-nJZ6P",
        "colab_type": "code",
        "outputId": "b0bd2ca3-6f67-40c3-d01b-976dd7ad3dd5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 86
        }
      },
      "source": [
        "start_time = time.time()\n",
        "for i in range(loop_count):\n",
        "    onnx_predicted_label = sess.run(None, feed)[0].argmax()\n",
        "print(\"ONNX inferences with %s second in average\" %((time.time() - start_time) / loop_count))\n",
        "\n",
        "print('ONNX predicted value:', onnx_predicted_label)\n",
        "if (expected_label == onnx_predicted_label):\n",
        "  print('Correct prediction !')\n",
        "else:\n",
        "  print('Wrong prediction !')\n",
        "\n",
        "if (predicted_label == onnx_predicted_label):\n",
        "  print(\"The ONNX's and keras' prediction are matching !\")\n",
        "else:\n",
        "  print(\"The ONNX's and keras' prediction does not match !\")\n"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "ONNX inferences with 0.0009166479110717773 second in average\n",
            "ONNX predicted value: 8\n",
            "Correct prediction !\n",
            "The ONNX's and keras' prediction are matching !\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}